# -*- coding": "utf-8 -*- ",
# @Time ": "2021/3/26 20:54 ",
# @Author ": "Dong ",
# @File ": "demo_one.py",

# (.+): (.+)$
# "$1": "$2",

import json
from urllib.parse import urlencode

import requests
from bs4 import BeautifulSoup
from requests.exceptions import RequestException

def parse_one_page(offset,keyword):
    data = {
        "aid": offset,
        "app_name": "web_search",
        "offset": "0",
        "format": "json",
        "keyword": keyword,
        "autoload": "true",
        "count": "20",
        "en_qc": "1",
        "cur_tab": "1",
        "from": "search_tab",
        "pd": "synthesis",
        "timestamp": "1616768524018",
        "_signature": "_02B4Z6wo00f01wKt.DAAAIDAslwQ - Rf0ivsCifiAAKEA1RgoVTSnuoHFRMg7cNrQn7PCYo4R6U1yvtkpbvXxQAved7e.8eNbyIsh2XdjKQ64r3sCUExOG5AFx1EQUHJCpyh7XSYjL - O3zUc212",
    }
    url = 'https://www.toutiao.com/api/search/content/?' + urlencode(data)
    try:
        response = requests.get(url)
        if response.status_code == 200:
            return response.text
    except RequestException:
        return None

def parse_page_index(html):
    # 接收的是json数据
    data = json.loads(html)
    if data and 'data' in data.keys():
        for item in data.get('data'):
            yield item.get('article_url')

def get_detail(url):
    try:
        response = requests.get(url)
        if response.status_code == 200:
            return response.text
    except RequestException:
        return None

def parse_detail(html):
    soup = BeautifulSoup(html,'lxml')
    print(soup)
    title = soup.select('title')[0].get_txt()
    print(title)

def main(offset,keyword):
    html = parse_one_page(offset,keyword)
    for item in parse_page_index(html):
        # html2 = get_detail(item)
        print(item)
        # if html2:
        #     parse_detail(html2)




# if __name__ == '__main__':
#     main(24,'街拍')

data = requests.get('https://www.toutiao.com/a6817621907341312515/')
print(data.text)
